#!/usr/bin/env python
# http://code.google.com/beaconbits/
# This code is released under license GPL2
# http://www.gnu.org/licenses/gpl-2.0.html


import sys
import subprocess
import argparse
import datetime
import time
import math
import redis

options = None

''' a few embedded algorithms that will be moved out as selectable at the command line'''
def compensated_variance(data):
    # sourced from HTTP://en.wikipedia.org/w/index.php?title=Algorithms_for_calculating_variance
    n = 0
    sum1 = 0
    for x in data:
        n = n + 1
        sum1 = sum1 + int(x)
    mean = sum1/n
 
    sum2 = 0
    sum3 = 0
    for x in data:
        sum2 = sum2 + (int(x) - mean)**2
        sum3 = sum3 + (int(x) - mean)
    variance = (sum2 - sum3**2/n)/(n - 1)
    return variance

def online_variance(data):
    # sourced from HTTP://en.wikipedia.org/w/index.php?title=Algorithms_for_online_variance    
    n = 0
    mean = 0
    M2 = 0
 
    for x in data:
        n = n + 1
        delta = int(x) - mean
        mean = mean + delta/n
        M2 = M2 + delta*(int(x) - mean)
 
    variance_n = M2/n
    variance = M2/(n - 1)
    return (variance, variance_n)

def population_fix(data):
        goods=[]
        diff = int(data[1]) - int(data[0])
        for each in data:
                workvalue = int(each) - int(diff)
                goods.append(workvalue)
                diff = each
        goods.pop(0)
        return(goods)



acceptable = ['All','Top']
worklist = []
tempset = []
magic_values = 15,29,30,31,59,60,61,89,90,91,119,120,121,239,240,241,299,300,301,400,514,600,720,900,1200,1800,3600
toplist1 = ()
toplist = []

def main():
        global options
        
        parser = argparse.ArgumentParser(description='Beacon bits is a time series analyzer for beacons. Requires Redis server running on default port and writes to db0.')

        # Either All or Top (for now)
        parser.add_argument('-t','--type',help='Specify either top or all as the type of evaluation selected')
        args = parser.parse_args()
        print args

        if args.type == 'top' or args.type == 'all':
            print 'src_ip','dst_ip','dst_port','set_date','pair_count','mean','duration_est','std_dev','compvar','onlinevar-0','onlinevar-1','src_count','dst_count','visitors','mvalue'
        else:
            sys.exit(-1)

        '''open a connection to the local redis database'''        
        r = redis.StrictRedis(host='localhost', port=6379, db=0)

        # The parsed query would allow for SET:192.168.1.1:* so that you can subdivide the search
        # If the query is seeking ALL then deliver all within reason
        roundone = r.keys('SET:*')

        for each in roundone:
                newcount = r.scard(each)
                if newcount <= 4:
                        r.delete(each)
                else:
                        worklist.append(each)

        #call the members
        for each in worklist:
                # zeroing math values needed in this routine and tab_queue
                tab_queue = 0
                cummulative_value = 0
                inner_count = 0
                visitors = 0

                # timing events by count stored in the pair_count
                pair_count = r.scard(each)

                # store the each value into our tempset
                tempset = r.sort(each, alpha=True)
                
                ''' HIGH MATH, statistical functions here'''
                compvar = compensated_variance(population_fix(tempset))
                onlinevar = online_variance(population_fix(tempset))

                # breakout the set between brackets to get count and perform lookups.
                sets_sub = each.split(':')
                set_src_ip = sets_sub[1]
                set_dst_ip = sets_sub[2]
                set_dst_port = sets_sub[3]
                set_date = sets_sub[4]
                src_count = r.get('ip_src:'+set_src_ip)
                dst_count = r.get('ip_dst:'+set_dst_ip)

                # the difference between the first and next value is our working initial mean value
                first_seen = tempset[0]
                next_value = tempset[1]
                # Redis stores as strings, convert to integers
                work_value = int(next_value) - int(first_seen)
        
                # evaluate each SET:*  required for one of our algorithms and few unusual calculations where inline beats evaluation of the array or set
                # tempset contains the SET:1.1.1.1:2.2.2.2:80:multi
                for tempset_sub in tempset:
                        # tepset_sub is each value, pull from the list
                        if tempset_sub == first_seen:
                                # store the value
                                tab_queue = int(tempset_sub)
                                # fix cummulative_value to reflect the first number
                                cummulative_value = work_value
                                # need this for singular math build and standard deviation
                                inner_count = 1
                                # reset the standard deviation values to ensure clean values
                                std_dev = 0
                                std_dev_prep = 0
                        else:
                                # don't move these evaluations to a function just yet
                                # Store tempset_sub as integer for subtraction
                                # Established work_value and current tempset_sub display them
                                key_value = int(tempset_sub) - tab_queue                                                                
                                # create a cumulative value based on key_value
                                cummulative_value = cummulative_value + key_value
                                # store the temset_sub as tab_queue for the next iteration
                                tab_queue = int(tempset_sub)
                                # population math to calculate the standard Deviation
                                #working mean is the cummulative_value divided by the current inner_count
                                work_mean = cummulative_value
                                # increment the inner count to have std_ddev work
                                inner_count = inner_count + 1
                                # A very deviant implementation of standard deviation prep and increment
                                # May have 'divide by zero' errors and it should be evaluated
                                # This variant of standard deviation is not in a pure statistical implimentaiton
                                # Do not recommend using it it outside of the method imployed here
                                std_dev_prep = math.sqrt((key_value + work_mean)**2)                                
                                try:
                                        std_dev = math.sqrt((1.0/(work_mean-1))*(std_dev_prep/inner_count))
                                except:
                                        print 'failure for',each, work_mean, pair_count

                ''' LOW MATH and intial condition testing '''
                ## this section is our LOW MATH, ratios, means etc. useful in our evaluation of beacons
                # establish the overall mean              
                mean = cummulative_value / pair_count
                # hours estimate for beacon
                # a simple estimate of minutes of persistence
                duration_est = (mean * pair_count) / 60
                if duration_est == 0:
                        duration_est = 1
                # The SYN count for the IP source and IP destination and divide it by how many times we see hits at the IP destination and call that visitors
                visitors =   int(dst_count) / int(pair_count)
                
                ''' Good traffic and short session shall not pass this point '''
                ''' Ensure traffic time exceeds 900 seconds and has less then 5 unique source IPs to a given destination or less'''
                ''' These values will be variables passed by the user at some point'''
                # Remove from evaluation any traffic that is less then 15 minutes
                if duration_est >= 900 and visitors <= 5:
    
                    ''' LOW MATH CONDITIONAL TESTING'''
                    ## conditional test of ratios and values
                    # The M value represent very specific intervals in the mean, add to our top list
                    if int(mean) in magic_values:
                        mvalue = True
                        # create a tuple with our goods
                        toplist1 = (set_src_ip,set_dst_ip,set_dst_port,set_date,pair_count,mean,duration_est,std_dev,compvar,onlinevar[0],onlinevar[1],src_count,dst_count,visitors,mvalue)
                        toplist.append(toplist1)
                    else:
                            mvalue = ''
    
                    # Evaluate for variance type and add to toplist for printing at the end
                    if compvar <= 300:
                        toplist2 = (set_src_ip,set_dst_ip,set_dst_port,set_date,pair_count,mean,duration_est,std_dev,compvar,onlinevar[0],onlinevar[1],src_count,dst_count,visitors,mvalue)
                        if toplist2 not in toplist:
                            toplist.append(toplist2)
                    else:
                        if int(compvar) / duration_est  <= 140:
                            toplist2 = (set_src_ip,set_dst_ip,set_dst_port,set_date,pair_count,mean,duration_est,std_dev,compvar,onlinevar[0],onlinevar[1],src_count,dst_count,visitors,mvalue)
                            if toplist2 not in toplist:
                                toplist.append(toplist2)
                    if std_dev <= float(.005):
                        toplist2 = (set_src_ip,set_dst_ip,set_dst_port,set_date,pair_count,mean,duration_est,std_dev,compvar,onlinevar[0],onlinevar[1],src_count,dst_count,visitors,mvalue)
                        if toplist2 not in toplist:
                            toplist.append(toplist2)                   

                # flush key_value
                key_value = []
                
                ''' If you select all you will get all '''
                # If you select '-t all' 
                if args.type == 'all':
                    print set_src_ip,'\t',set_dst_ip,'\t',set_dst_port,'\t',set_date,'\t',pair_count,'\t',mean,'\t',duration_est,'\t',std_dev,'\t',compvar,'\t',onlinevar[0],'\t',onlinevar[1],'\t',src_count,' \t',dst_count,'\t',visitors,'\t',mvalue

        ''' printing top values'''
        # sor the top list
        toplist.sort()
        # outter unpacking of the tuple
        for each in toplist:
            #print str(each)
            print format(each)

        print "Finished"
if __name__ == '__main__':
        main()




